package fetcher

import (
    "bufio"
    "fmt"
    "golang.org/x/net/html/charset"
    "golang.org/x/text/encoding"
    "golang.org/x/text/encoding/unicode"
    "golang.org/x/text/transform"
    "io/ioutil"
    "log"
    "net/http"
    "time"
)

// 定义一个每隔100毫秒自动填充时间数据的的channel
var rateLimiter = time.Tick(10 * time.Millisecond)

func Fetch(url string) ([]byte, error) {
    // 格式:2018-08-04 17:58:29.4974511 +0800 CST m=+1.113025101
    // fmt.Println(<-rateLimiter)
    
    <-rateLimiter
    
    req, _ := http.NewRequest(http.MethodGet, url, nil)
    req.Header.Set("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/68.0.3440.84 Safari/537.36")
    
    var httpClient = http.Client{}
    resp, err := httpClient.Do(req)
    if err != nil {
        return nil, err
    }
    
    defer resp.Body.Close()
    
    if resp.StatusCode != 200 {
        return nil, fmt.Errorf("抓取出错了, 返回码[%d]", resp.StatusCode)
    }
    
    bufBody := bufio.NewReader(resp.Body)
    utf8Reader := transform.NewReader(bufBody, determineEncoding(bufBody).NewDecoder())
    body, err := ioutil.ReadAll(utf8Reader)
    
    return body, err
}

// 根据reader分析编码
func determineEncoding(r *bufio.Reader) encoding.Encoding {
    bytes, err := bufio.NewReader(r).Peek(1024)
    if err != nil {
        log.Printf("Fetcher error: %v", err)
        return unicode.UTF8
    }
    enco, _, _ := charset.DetermineEncoding(bytes, "")
    return enco
}
